# -*- coding: gb2312 -*-
from pymongo import MongoClient
from WindPy import *

import time
import math


class StockDailyFactorGeneralFunctions:
    def __init__(self, fillType='B', DBIP='localhost', DBPort=27017, DBName = 'stocks',  colSysData = 'sys_data'):
        self.fillType = fillType

        self.DBIP = DBIP
        self.DBPort = DBPort
        self.DBName = DBName
        self.stockIdList = []
        self.periodTransDateList = []
        self.colSysData = colSysData
        w.start()

    def retrieveStockIdList(self, endDateStr):
        resStockID = w.wset('sectorconstituent', 'date=' + endDateStr + ';sectorid=a001010100000000;field=wind_code')
        return resStockID.Data[0]

    def retrieveIndexIdList(self):
        resIndicesID = ['000001.SH', '399001.SZ', '399006.SZ', '000300.SH', '000016.SH', '000905.SH', '399005.SZ', '881001.WI']
        return resIndicesID



    def writeBaseDataToMongoDB(self, startDateStr, endDateStr, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, stockSplitGrpNum = 10):

        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName
        fillType = self.fillType


        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()
        totalStockIdNum = len(stockIDdata)

        ###############################################################################################################
        #this shows stock number to retrieve in one group
        ###############################################################################################################

        splitStocksNum = math.floor(totalStockIdNum / stockSplitGrpNum)

        client = MongoClient(DBIP, DBPort)

        db = client[DBName]

        collection_data = db[colNameStr]

        ###############################################################################################################
        #  we use the selected stock number to differentiate stocks data and index data, if the number is greater than
        # 10, we treat it as a list stocks, otherwise as a list of indices
        ###############################################################################################################
        if totalStockIdNum>10:
            for i in range(0, stockSplitGrpNum):
                stockIdOneLoopList = []
                for j in range(0, splitStocksNum):
                    stockIdOneLoopList.append(i*splitStocksNum + j)
                oneStockSearchStr = ''
                for aStockID in stockIdOneLoopList:
                    oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Fill=Previous;PriceAdj=' + fillType)

                batchStockNum = len(resWindData.Data)
                transDataTime = resWindData.Times

                for k in range(0, batchStockNum):
                    aData = []
                    aDateDict = {}
                    aDateDict['StockID'] = stockIDdata[i * splitStocksNum + k]
                    totalTransDaysNum = len(transDataTime)
                    for l in range(0, totalTransDaysNum):
                        aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                    aData.append(aDateDict)
                    collection_data.insert(aData)
                print('Finished writing' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)

            ###############################################################################################################
            # After the looped retrieval, there could be still some stocks unretrieved, i.e., looped retrieved stocks may
            # not be the full set of available stocks, therefore we should locate the unretrieved stocks
            # by using mod function, we can check if remaining stocks is the full set of the available stock set
            # if not, we locate the remains
            ###############################################################################################################
            if (totalStockIdNum % stockSplitGrpNum) != 0:
                stockIdRemainsList = []
                for m in range((stockSplitGrpNum-1)*splitStocksNum + splitStocksNum, totalStockIdNum):
                    stockIdRemainsList.append(m)
                oneStockSearchStr = ''

                for aStockID in stockIdRemainsList:
                    oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                    resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Fill=Previous;PriceAdj=' + fillType)
                batchStockNum = len(resWindData.Data)
                transDataTime = resWindData.Times

                for k in range(0, batchStockNum):
                    aData = []
                    aDateDict = {}
                    aDateDict['StockID'] = stockIDdata[(stockSplitGrpNum-1)*splitStocksNum + splitStocksNum + k]
                    for l in range(0, totalTransDaysNum):
                        aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                    aData.append(aDateDict)
                    collection_data.insert(aData)
                print('Finished writing remaining' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)

        ###############################################################################################################
        #  below is to process index data
        ###############################################################################################################
        else:
            stockIdList = []
            for m in range(0, totalStockIdNum):
                stockIdList.append(m)
            oneStockSearchStr = ''

            for aStockID in stockIdList:
                oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr,
                                    'Fill=Previous;PriceAdj=' + fillType)
            batchStockNum = len(resWindData.Data)
            transDataTime = resWindData.Times
            totalTransDaysNum = len(transDataTime)
            for k in range(0, batchStockNum):
                aData = []
                aDateDict = {}
                aDateDict['StockID'] = stockIDdata[k]
                for l in range(0, totalTransDaysNum):
                    aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                aData.append(aDateDict)
                collection_data.insert(aData)
            print('Finished writing remaining' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)
        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, currentDateStr = endDateStr)


    def writeBaseQuarterDataToMongoDB(self, startDateStr, endDateStr, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, stockSplitGrpNum = 10):

        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName
        fillType = self.fillType


        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()
        totalStockIdNum = len(stockIDdata)

        ###############################################################################################################
        #this shows stock number to retrieve in one group
        ###############################################################################################################

        splitStocksNum = math.floor(totalStockIdNum / stockSplitGrpNum)

        client = MongoClient(DBIP, DBPort)

        db = client[DBName]

        collection_data = db[colNameStr]

        ###############################################################################################################
        #  we use the selected stock number to differentiate stocks data and index data, if the number is greater than
        # 10, we treat it as a list stocks, otherwise as a list of indices
        ###############################################################################################################
        if totalStockIdNum>10:
            for i in range(0, stockSplitGrpNum):
                stockIdOneLoopList = []
                for j in range(0, splitStocksNum):
                    stockIdOneLoopList.append(i*splitStocksNum + j)
                oneStockSearchStr = ''
                for aStockID in stockIdOneLoopList:
                    oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Period=Q;Fill=Previous;PriceAdj=' + fillType)

                batchStockNum = len(resWindData.Data)
                transDataTime = resWindData.Times

                for k in range(0, batchStockNum):
                    aData = []
                    aDateDict = {}
                    aDateDict['StockID'] = stockIDdata[i * splitStocksNum + k]
                    totalTransDaysNum = len(transDataTime)
                    for l in range(0, totalTransDaysNum):
                        aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                    aData.append(aDateDict)
                    collection_data.insert(aData)
                print('Finished writing' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)

            ###############################################################################################################
            # After the looped retrieval, there could be still some stocks unretrieved, i.e., looped retrieved stocks may
            # not be the full set of available stocks, therefore we should locate the unretrieved stocks
            # by using mod function, we can check if remaining stocks is the full set of the available stock set
            # if not, we locate the remains
            ###############################################################################################################
            if (totalStockIdNum % stockSplitGrpNum) != 0:
                stockIdRemainsList = []
                for m in range((stockSplitGrpNum-1)*splitStocksNum + splitStocksNum, totalStockIdNum):
                    stockIdRemainsList.append(m)
                oneStockSearchStr = ''

                for aStockID in stockIdRemainsList:
                    oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                    resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Period=Q;Fill=Previous;PriceAdj=' + fillType)
                batchStockNum = len(resWindData.Data)
                transDataTime = resWindData.Times

                for k in range(0, batchStockNum):
                    aData = []
                    aDateDict = {}
                    aDateDict['StockID'] = stockIDdata[(stockSplitGrpNum-1)*splitStocksNum + splitStocksNum + k]
                    for l in range(0, totalTransDaysNum):
                        aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                    aData.append(aDateDict)
                    collection_data.insert(aData)
                print('Finished writing remaining' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)

        ###############################################################################################################
        #  below is to process index data
        ###############################################################################################################
        else:
            stockIdList = []
            for m in range(0, totalStockIdNum):
                stockIdList.append(m)
            oneStockSearchStr = ''

            for aStockID in stockIdList:
                oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr,
                                    'Period=Q;Fill=Previous;PriceAdj=' + fillType)
            batchStockNum = len(resWindData.Data)
            transDataTime = resWindData.Times
            totalTransDaysNum = len(transDataTime)
            for k in range(0, batchStockNum):
                aData = []
                aDateDict = {}
                aDateDict['StockID'] = stockIDdata[k]
                for l in range(0, totalTransDaysNum):
                    aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                aData.append(aDateDict)
                collection_data.insert(aData)
            print('Finished writing remaining' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)
        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, currentDateStr = endDateStr)


    def writeBaseReciprocalDataToMongoDB(self, startDateStr, endDateStr, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, stockSplitGrpNum = 10):

        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName
        fillType = self.fillType


        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()
        totalStockIdNum = len(stockIDdata)

        ###############################################################################################################
        #this shows stock number to retrieve in one group
        ###############################################################################################################

        splitStocksNum = math.floor(totalStockIdNum / stockSplitGrpNum)

        client = MongoClient(DBIP, DBPort)

        db = client[DBName]

        collection_data = db[colNameStr]

        ###############################################################################################################
        #  we use the selected stock number to differentiate stocks data and index data, if the number is greater than
        # 10, we treat it as a list stocks, otherwise as a list of indices
        ###############################################################################################################
        if totalStockIdNum>10:
            for i in range(0, stockSplitGrpNum):
                stockIdOneLoopList = []
                for j in range(0, splitStocksNum):
                    stockIdOneLoopList.append(i*splitStocksNum + j)
                oneStockSearchStr = ''
                for aStockID in stockIdOneLoopList:
                    oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Fill=Previous;PriceAdj=' + fillType)

                batchStockNum = len(resWindData.Data)
                transDataTime = resWindData.Times

                for k in range(0, batchStockNum):
                    aData = []
                    aDateDict = {}
                    aDateDict['StockID'] = stockIDdata[i * splitStocksNum + k]
                    totalTransDaysNum = len(transDataTime)
                    for l in range(0, totalTransDaysNum):
                        try:
                            aDateDict[transDataTime[l].strftime('%Y%m%d')] = 1 / resWindData.Data[k][l]
                        except:
                            aDateDict[transDataTime[l].strftime('%Y%m%d')] = 0.0
                    aData.append(aDateDict)
                    collection_data.insert(aData)
                print('Finished writing' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)

            ###############################################################################################################
            # After the looped retrieval, there could be still some stocks unretrieved, i.e., looped retrieved stocks may
            # not be the full set of available stocks, therefore we should locate the unretrieved stocks
            # by using mod function, we can check if remaining stocks is the full set of the available stock set
            # if not, we locate the remains
            ###############################################################################################################
            if (totalStockIdNum % stockSplitGrpNum) != 0:
                stockIdRemainsList = []
                for m in range((stockSplitGrpNum-1)*splitStocksNum + splitStocksNum, totalStockIdNum):
                    stockIdRemainsList.append(m)
                oneStockSearchStr = ''

                for aStockID in stockIdRemainsList:
                    oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                    resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Fill=Previous;PriceAdj=' + fillType)
                batchStockNum = len(resWindData.Data)
                transDataTime = resWindData.Times

                for k in range(0, batchStockNum):
                    aData = []
                    aDateDict = {}
                    aDateDict['StockID'] = stockIDdata[(stockSplitGrpNum-1)*splitStocksNum + splitStocksNum + k]
                    for l in range(0, totalTransDaysNum):
                        try:
                            aDateDict[transDataTime[l].strftime('%Y%m%d')] = 1 / float(resWindData.Data[k][l])
                        except :
                            aDateDict[transDataTime[l].strftime('%Y%m%d')] = 0.0
                    aData.append(aDateDict)
                    collection_data.insert(aData)
                print('Finished writing remaining' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)

        ###############################################################################################################
        #  below is to process index data
        ###############################################################################################################
        else:
            stockIdList = []
            for m in range(0, totalStockIdNum):
                stockIdList.append(m)
            oneStockSearchStr = ''

            for aStockID in stockIdList:
                oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr,
                                    'Fill=Previous;PriceAdj=' + fillType)
            batchStockNum = len(resWindData.Data)
            transDataTime = resWindData.Times
            totalTransDaysNum = len(transDataTime)
            for k in range(0, batchStockNum):
                aData = []
                aDateDict = {}
                aDateDict['StockID'] = stockIDdata[k]
                for l in range(0, totalTransDaysNum):
                    try:
                        aDateDict[transDataTime[l].strftime('%Y%m%d')] = 1 / float(resWindData.Data[k][l])
                    except:
                        aDateDict[transDataTime[l].strftime('%Y%m%d')] = 0.0
                aData.append(aDateDict)
                collection_data.insert(aData)
            print('Finished writing remaining' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)
        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, currentDateStr = endDateStr)




    def writeBaseBuySellDataToMongoDB(self, startDateStr, endDateStr, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, traderTypeStr, stockSplitGrpNum = 10):

        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName
        fillType = self.fillType



        stockIDdata = self.retrieveStockIdList(endDateStr)
        totalStockIdNum = len(stockIDdata)

        ###############################################################################################################
        #this shows stock number to retrieve in one group
        ###############################################################################################################

        splitStocksNum = math.floor(totalStockIdNum / stockSplitGrpNum)

        client = MongoClient(DBIP, DBPort)

        db = client[DBName]

        collection_data = db[colNameStr]

        ###############################################################################################################
        #  we use the selected stock number to differentiate stocks data and index data, if the number is greater than
        # 10, we treat it as a list stocks, otherwise as a list of indices
        ###############################################################################################################

        for i in range(0, stockSplitGrpNum):
            stockIdOneLoopList = []
            for j in range(0, splitStocksNum):
                stockIdOneLoopList.append(i*splitStocksNum + j)
            oneStockSearchStr = ''
            for aStockID in stockIdOneLoopList:
                oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
            resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'unit=1;traderType=' + traderTypeStr + ';PriceAdj=' + fillType)

            batchStockNum = len(resWindData.Data)
            transDataTime = resWindData.Times

            for k in range(0, batchStockNum):
                aData = []
                aDateDict = {}
                aDateDict['StockID'] = stockIDdata[i * splitStocksNum + k]
                totalTransDaysNum = len(transDataTime)
                for l in range(0, totalTransDaysNum):
                    aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                aData.append(aDateDict)
                collection_data.insert(aData)
            print('Finished writing' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)

        ###############################################################################################################
        # After the looped retrieval, there could be still some stocks unretrieved, i.e., looped retrieved stocks may
        # not be the full set of available stocks, therefore we should locate the unretrieved stocks
        # by using mod function, we can check if remaining stocks is the full set of the available stock set
        # if not, we locate the remains
        ###############################################################################################################
        if (totalStockIdNum % stockSplitGrpNum) != 0:
            stockIdRemainsList = []
            for m in range((stockSplitGrpNum-1)*splitStocksNum + splitStocksNum, totalStockIdNum):
                stockIdRemainsList.append(m)
            oneStockSearchStr = ''

            for aStockID in stockIdRemainsList:
                oneStockSearchStr = oneStockSearchStr + str(stockIDdata[aStockID]) + ', '
                resWindData = w.wsd(oneStockSearchStr, windFactorNameStr, startDateStr, endDateStr, 'unit=1;traderType=' + traderTypeStr + ';Fill=Previous;PriceAdj=' + fillType)
            batchStockNum = len(resWindData.Data)
            transDataTime = resWindData.Times

            for k in range(0, batchStockNum):
                aData = []
                aDateDict = {}
                aDateDict['StockID'] = stockIDdata[(stockSplitGrpNum-1)*splitStocksNum + splitStocksNum + k]
                for l in range(0, totalTransDaysNum):
                    aDateDict[transDataTime[l].strftime('%Y%m%d')] = resWindData.Data[k][l]
                aData.append(aDateDict)
                collection_data.insert(aData)
            print('Finished writing remaining' + dbUpdateDateFieldAliasStr + 'of stocks:' + oneStockSearchStr)


        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, currentDateStr = endDateStr)




    def updateDataToMongoDB(self, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, endDateStr = time.strftime('%Y-%m-%d')):
        fillType = self.fillType
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName

        startDateStr = self.retrieveDataUpdateDate(dbUpdateDateFiledStr)

        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()

        totalStockIdNum = len(stockIDdata)

        stockIdSearchStr = ''

        for n in range(0, totalStockIdNum):
            stockIdSearchStr = stockIdSearchStr + stockIDdata[n] + ', '
        resWindData = w.wsd(stockIdSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Fill=Previous;PriceAdj=' + fillType)

        transDataTime = resWindData.Times
        if (str(resWindData.Data[0][0]).find('No data') != -1):
            print('Currently it is NOT in a transacation day and data is up to date, no more data to be updated by now...')
            return ()

        totalTransDaysNum = len(transDataTime)

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_data = db[colNameStr]
        resWindDataIdList = resWindData.Data
        if (totalTransDaysNum > 1):
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFactorDataFloat = float(resWindDataIdList[n][l])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataFloat}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(
                        stockIDdata[n]) + ', on the day of ' + aDateStr)
        else:
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFactorDataFloat = float(resWindDataIdList[l][n])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataFloat}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(stockIDdata[n]) + ', on the day of ' + aDateStr)
        print('Finished updating' + dbUpdateDateFieldAliasStr + 'of all stocks of:' + stockIdSearchStr)

        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)


    def updateStringDataToMongoDB(self, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, endDateStr = time.strftime('%Y-%m-%d')):
        fillType = self.fillType
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName

        startDateStr = self.retrieveDataUpdateDate(dbUpdateDateFiledStr)

        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()

        totalStockIdNum = len(stockIDdata)

        stockIdSearchStr = ''

        for n in range(0, totalStockIdNum):
            stockIdSearchStr = stockIdSearchStr + stockIDdata[n] + ', '
        resWindData = w.wsd(stockIdSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Fill=Previous;PriceAdj=' + fillType)

        transDataTime = resWindData.Times
        if (str(resWindData.Data[0][0]).find('No data') != -1):
            print('Currently it is NOT in a transacation day and data is up to date, no more data to be updated by now...')
            return ()

        totalTransDaysNum = len(transDataTime)

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_data = db[colNameStr]
        resWindDataIdList = resWindData.Data
        if (totalTransDaysNum > 1):
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFactorDataStr = str(resWindDataIdList[n][l])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataStr}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(
                        stockIDdata[n]) + ', on the day of ' + aDateStr)
        else:
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFactorDataStr = str(resWindDataIdList[l][n])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataStr}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(stockIDdata[n]) + ', on the day of ' + aDateStr)
        print('Finished updating' + dbUpdateDateFieldAliasStr + 'of all stocks of:' + stockIdSearchStr)

        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)


    def updateQuarterIssuingDatesToMongoDB(self, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, dateList):
        fillType = self.fillType
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName

        quarterNum = len(dateList)
        startDateStr = dateList[0].strftime('%Y-%m-%d')
        endDateStr = dateList[quarterNum-1].strftime('%Y-%m-%d')


        stockIDdata = self.retrieveStockIdList(endDateStr)

        totalStockIdNum = len(stockIDdata)

        stockIdSearchStr = ''

        for n in range(0, totalStockIdNum):
            stockIdSearchStr = stockIdSearchStr + stockIDdata[n] + ', '
        resWindData = w.wsd(stockIdSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Period=Q;Fill=Previous;PriceAdj=' + fillType)

        transDataTime = resWindData.Times
        if (str(resWindData.Data[0][0]).find('No data') != -1):
            print('Currently it is NOT in a transacation day and data is up to date, no more data to be updated by now...')
            return ()

        totalTransDaysNum = len(transDataTime)

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_data = db[colNameStr]
        resWindDataIdList = resWindData.Data
        if (totalTransDaysNum > 1):
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                aFactorDataStr = '20100331'
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    try:
                        aFactorDataStr = resWindDataIdList[n][l].strftime('%Y%m%d')
                    except:
                        print("blank or wrong date data....")

                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataStr}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(
                        stockIDdata[n]) + ', on the day of ' + aDateStr)
        else:
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFactorDataStr = str(resWindDataIdList[l][n])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataStr}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(stockIDdata[n]) + ', on the day of ' + aDateStr)
        print('Finished updating' + dbUpdateDateFieldAliasStr + 'of all stocks of:' + stockIdSearchStr)

        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)


    def updateQuarterDataToMongoDB(self, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, dateList):
        fillType = self.fillType
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName

        quarterNum = len(dateList)
        startDateStr = dateList[0].strftime('%Y-%m-%d')
        endDateStr = dateList[quarterNum-1].strftime('%Y-%m-%d')



        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()

        totalStockIdNum = len(stockIDdata)

        stockIdSearchStr = ''

        for n in range(0, totalStockIdNum):
            stockIdSearchStr = stockIdSearchStr + stockIDdata[n] + ', '
        resWindData = w.wsd(stockIdSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Period=Q;Fill=Previous;PriceAdj=' + fillType)

        transDataTime = resWindData.Times
        if (str(resWindData.Data[0][0]).find('No data') != -1):
            print('Currently it is NOT in a transacation day and data is up to date, no more data to be updated by now...')
            return ()

        totalTransDaysNum = len(transDataTime)

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_data = db[colNameStr]
        resWindDataIdList = resWindData.Data
        if (totalTransDaysNum > 1):
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFactorDataFloat = float(resWindDataIdList[n][l])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataFloat}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(
                        stockIDdata[n]) + ', on the day of ' + aDateStr)
        else:
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFactorDataFloat = float(resWindDataIdList[l][n])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataFloat}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(stockIDdata[n]) + ', on the day of ' + aDateStr)
        print('Finished updating' + dbUpdateDateFieldAliasStr + 'of all stocks of:' + stockIdSearchStr)

        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)


    def updateReciprocalDataToMongoDB(self, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, endDateStr = time.strftime('%Y-%m-%d')):
        fillType = self.fillType
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName

        startDateStr = self.retrieveDataUpdateDate(dbUpdateDateFiledStr)

        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()

        totalStockIdNum = len(stockIDdata)

        stockIdSearchStr = ''

        for n in range(0, totalStockIdNum):
            stockIdSearchStr = stockIdSearchStr + stockIDdata[n] + ', '
        resWindData = w.wsd(stockIdSearchStr, windFactorNameStr, startDateStr, endDateStr, 'Fill=Previous;PriceAdj=' + fillType)

        transDataTime = resWindData.Times
        if (str(resWindData.Data[0][0]).find('No data') != -1):
            print('Currently it is NOT in a transacation day and data is up to date, no more data to be updated by now...')
            return ()

        totalTransDaysNum = len(transDataTime)

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_data = db[colNameStr]
        resWindDataIdList = resWindData.Data
        if (totalTransDaysNum > 1):
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    try:
                        aFactorDataFloat = 1/float(resWindDataIdList[n][l])
                    except:
                        aFactorDataStr = '0'
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataFloat}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(
                        stockIDdata[n]) + ', on the day of ' + aDateStr)
        else:
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    try:
                        aFactorDataFloat = 1/float(resWindDataIdList[n][l])
                    except:
                        aFactorDataStr = '0'
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFactorDataFloat}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(stockIDdata[n]) + ', on the day of ' + aDateStr)
        print('Finished updating' + dbUpdateDateFieldAliasStr + 'of all stocks of:' + stockIdSearchStr)

        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)



    def updateBuySellDataToMongoDB(self, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr, traderTypeStr, endDateStr = time.strftime('%Y-%m-%d')):
        fillType = self.fillType
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName

        startDateStr = self.retrieveDataUpdateDate(dbUpdateDateFiledStr)

        if (colNameStr != 'indices_close'):
            stockIDdata = self.retrieveStockIdList(endDateStr)
        else:
            stockIDdata = self.retrieveIndexIdList()

        totalStockIdNum = len(stockIDdata)

        stockIdSearchStr = ''

        for n in range(0, totalStockIdNum):
            stockIdSearchStr = stockIdSearchStr + stockIDdata[n] + ', '
        resWindData = w.wsd(stockIdSearchStr, windFactorNameStr, startDateStr, endDateStr, 'unit=1;traderType=' + traderTypeStr + ';PriceAdj=' + fillType)

        transDataTime = resWindData.Times
        if (str(resWindData.Data[0][0]).find('No data') != -1):
            print('Currently it is NOT in a transacation day and data is up to date, no more data to be updated by now...')
            return ()

        totalTransDaysNum = len(transDataTime)

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_data = db[colNameStr]
        resWindDataIdList = resWindData.Data
        if (totalTransDaysNum > 1):
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFloatFactorData = float(resWindDataIdList[n][l])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFloatFactorData}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(
                        stockIDdata[n]) + ', on the day of ' + aDateStr)
        else:
            for n in range(0, totalStockIdNum):
                aStockIdStr = str(stockIDdata[n])
                for l in range(0, totalTransDaysNum):
                    aDateStr = transDataTime[l].strftime('%Y%m%d')
                    aFloatFactorData = float(resWindDataIdList[l][n])
                    collection_data.update({'StockID': aStockIdStr}, {'$set': {aDateStr: aFloatFactorData}}, upsert=True)
                    print('finished updating' + dbUpdateDateFieldAliasStr + 'of stock:' + str(stockIDdata[n]) + ', on the day of ' + aDateStr)
        print('Finished updating' + dbUpdateDateFieldAliasStr + 'of all stocks of:' + stockIdSearchStr)

        self.updateDataUpdateDate(dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)






    def updateDataUpdateDate(self, strUpdatedKeyName, strUpdatedKeyAlias, currentDateStr = time.strftime('%Y-%m-%d')):
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName
        colSysData = self.colSysData

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_sys_data = db[colSysData]
        try:
            collection_sys_data.update({}, {'$set': {strUpdatedKeyName: currentDateStr}})
        except:
            print('Failed to update  date to MongoDB, please check.......')
            return()
        print('Successfully updated' + strUpdatedKeyAlias + 'into MongoDB.')

    ###################################################################################################################
    ### Obtain Data Update Date
    ###################################################################################################################
    def retrieveDataUpdateDate(self, strUpdateTimeKey):
        DBIP = self.DBIP
        DBPort = self.DBPort
        DBName = self.DBName
        colSysData = self.colSysData

        client = MongoClient(DBIP, DBPort)
        db = client[DBName]
        collection_sys_data = db[colSysData]


        sysDataRes = collection_sys_data.find({}, {strUpdateTimeKey: 1})

        dateListDict = sysDataRes[0]

        dataUpdateDate = dateListDict[strUpdateTimeKey]

        return dataUpdateDate


#obj = StockDailyFactorGeneralFunctions()
#obj2 = StockGeneralData

#baseStartDateStr = '2008-01-01'
#baseEndDateStr = '2017-05-01'
#windFactorNameStr = 'ev'
#colNameStr = 'ev'
#dbUpdateDateFiledStr = 'ev_update_date'
#dbUpdateDateFieldAliasStr = ' EV '


#obj.writeBaseDataToMongoDB(baseStartDateStr, baseEndDateStr, windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)


#obj.updateDataToMongoDB(windFactorNameStr, colNameStr, dbUpdateDateFiledStr, dbUpdateDateFieldAliasStr)
#obj.writeBaseEvToMongoDB(baseEndDateStr)
#obj.writeBaseMktFreeSharesToMongoDB(baseEndDateStr)
#obj.updateMktFreeSharesToMongoDB(baseEndDateStr)


#obj.updateEvToMongoDB(obj.retrieveDataUpdateDate('ev_update_date'))

